<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<html lang="en">

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <title>LCOV - code analysis - src/caffe/solvers/sgd_solver.cpp</title>
  <link rel="stylesheet" type="text/css" href="../../../gcov.css">
</head>

<body>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="title">LCOV - code coverage report</td></tr>
    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>

    <tr>
      <td width="100%">
        <table cellpadding=1 border=0 width="100%">
          <tr>
            <td width="10%" class="headerItem">Current view:</td>
            <td width="35%" class="headerValue"><a href="../../../index.html">top level</a> - <a href="index.html">src/caffe/solvers</a> - sgd_solver.cpp<span style="font-size: 80%;"> (source / <a href="sgd_solver.cpp.func-sort-c.html">functions</a>)</span></td>
            <td width="5%"></td>
            <td width="15%"></td>
            <td width="10%" class="headerCovTableHead">Hit</td>
            <td width="10%" class="headerCovTableHead">Total</td>
            <td width="15%" class="headerCovTableHead">Coverage</td>
          </tr>
          <tr>
            <td class="headerItem">Test:</td>
            <td class="headerValue">code analysis</td>
            <td></td>
            <td class="headerItem">Lines:</td>
            <td class="headerCovTableEntry">2</td>
            <td class="headerCovTableEntry">170</td>
            <td class="headerCovTableEntryLo">1.2 %</td>
          </tr>
          <tr>
            <td class="headerItem">Date:</td>
            <td class="headerValue">2020-09-11 22:25:26</td>
            <td></td>
            <td class="headerItem">Functions:</td>
            <td class="headerCovTableEntry">2</td>
            <td class="headerCovTableEntry">28</td>
            <td class="headerCovTableEntryLo">7.1 %</td>
          </tr>
          <tr>
            <td class="headerItem">Legend:</td>
            <td class="headerValueLeg">            Lines:
            <span class="coverLegendCov">hit</span>
            <span class="coverLegendNoCov">not hit</span>
</td>
            <td></td>
          </tr>
          <tr><td><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
        </table>
      </td>
    </tr>

    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
  </table>

  <table cellpadding=0 cellspacing=0 border=0>
    <tr>
      <td><br></td>
    </tr>
    <tr>
      <td>
<pre class="sourceHeading">          Line data    Source code</pre>
<pre class="source">
<a name="1"><span class="lineNum">       1 </span>            : #include &lt;string&gt;</a>
<span class="lineNum">       2 </span>            : #include &lt;vector&gt;
<span class="lineNum">       3 </span>            : 
<span class="lineNum">       4 </span>            : #include &quot;caffe/sgd_solvers.hpp&quot;
<span class="lineNum">       5 </span>            : #include &quot;caffe/util/hdf5.hpp&quot;
<span class="lineNum">       6 </span>            : #include &quot;caffe/util/io.hpp&quot;
<span class="lineNum">       7 </span>            : #include &quot;caffe/util/upgrade_proto.hpp&quot;
<span class="lineNum">       8 </span>            : 
<span class="lineNum">       9 </span>            : namespace caffe {
<span class="lineNum">      10 </span>            : 
<span class="lineNum">      11 </span>            : // Return the current learning rate. The currently implemented learning rate
<span class="lineNum">      12 </span>            : // policies are as follows:
<span class="lineNum">      13 </span>            : //    - fixed: always return base_lr.
<span class="lineNum">      14 </span>            : //    - step: return base_lr * gamma ^ (floor(iter / step))
<span class="lineNum">      15 </span>            : //    - exp: return base_lr * gamma ^ iter
<span class="lineNum">      16 </span>            : //    - inv: return base_lr * (1 + gamma * iter) ^ (- power)
<span class="lineNum">      17 </span>            : //    - multistep: similar to step but it allows non uniform steps defined by
<span class="lineNum">      18 </span>            : //      stepvalue
<span class="lineNum">      19 </span>            : //    - poly: the effective learning rate follows a polynomial decay, to be
<span class="lineNum">      20 </span>            : //      zero by the max_iter. return base_lr (1 - iter/max_iter) ^ (power)
<span class="lineNum">      21 </span>            : //    - sigmoid: the effective learning rate follows a sigmod decay
<span class="lineNum">      22 </span>            : //      return base_lr ( 1/(1 + exp(-gamma * (iter - stepsize))))
<span class="lineNum">      23 </span>            : //
<span class="lineNum">      24 </span>            : // where base_lr, max_iter, gamma, step, stepvalue and power are defined
<a name="25"><span class="lineNum">      25 </span>            : // in the solver parameter protocol buffer, and iter is the current iteration.</a>
<span class="lineNum">      26 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      27 </span><span class="lineNoCov">          0 : Dtype SGDSolver&lt;Dtype&gt;::GetLearningRate() {</span>
<span class="lineNum">      28 </span>            :   Dtype rate;
<span class="lineNum">      29 </span>            :   const string&amp; lr_policy = this-&gt;param_.lr_policy();
<span class="lineNum">      30 </span><span class="lineNoCov">          0 :   if (lr_policy == &quot;fixed&quot;) {</span>
<span class="lineNum">      31 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr();</span>
<span class="lineNum">      32 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;step&quot;) {</span>
<span class="lineNum">      33 </span><span class="lineNoCov">          0 :     CHECK_GT(this-&gt;param_.stepsize(), 0);</span>
<span class="lineNum">      34 </span><span class="lineNoCov">          0 :     this-&gt;current_step_ = this-&gt;iter_ / this-&gt;param_.stepsize();</span>
<span class="lineNum">      35 </span><span class="lineNoCov">          0 :     CHECK_GE(this-&gt;param_.gamma(), 0);</span>
<span class="lineNum">      36 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() *</span>
<span class="lineNum">      37 </span><span class="lineNoCov">          0 :         pow(this-&gt;param_.gamma(), this-&gt;current_step_);</span>
<span class="lineNum">      38 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;exp&quot;) {</span>
<span class="lineNum">      39 </span><span class="lineNoCov">          0 :     CHECK_GE(this-&gt;param_.gamma(), 0);</span>
<span class="lineNum">      40 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() * pow(this-&gt;param_.gamma(), this-&gt;iter_);</span>
<span class="lineNum">      41 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;inv&quot;) {</span>
<span class="lineNum">      42 </span><span class="lineNoCov">          0 :     CHECK_GE(this-&gt;param_.gamma(), 0);</span>
<span class="lineNum">      43 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() *</span>
<span class="lineNum">      44 </span><span class="lineNoCov">          0 :         pow(Dtype(1) + this-&gt;param_.gamma() * this-&gt;iter_,</span>
<span class="lineNum">      45 </span>            :             - this-&gt;param_.power());
<span class="lineNum">      46 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;multistep&quot;) {</span>
<span class="lineNum">      47 </span><span class="lineNoCov">          0 :     if (this-&gt;current_step_ &lt; this-&gt;param_.stepvalue_size() &amp;&amp;</span>
<span class="lineNum">      48 </span><span class="lineNoCov">          0 :           this-&gt;iter_ &gt;= this-&gt;param_.stepvalue(this-&gt;current_step_)) {</span>
<span class="lineNum">      49 </span><span class="lineNoCov">          0 :       this-&gt;current_step_++;</span>
<span class="lineNum">      50 </span><span class="lineNoCov">          0 :       LOG(INFO) &lt;&lt; &quot;MultiStep Status: Iteration &quot; &lt;&lt;</span>
<span class="lineNum">      51 </span>            :       this-&gt;iter_ &lt;&lt; &quot;, step = &quot; &lt;&lt; this-&gt;current_step_;
<span class="lineNum">      52 </span>            :     }
<span class="lineNum">      53 </span><span class="lineNoCov">          0 :     CHECK_GE(this-&gt;param_.gamma(), 0);</span>
<span class="lineNum">      54 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() *</span>
<span class="lineNum">      55 </span><span class="lineNoCov">          0 :         pow(this-&gt;param_.gamma(), this-&gt;current_step_);</span>
<span class="lineNum">      56 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;poly&quot;) {</span>
<span class="lineNum">      57 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() * pow(Dtype(1.) -</span>
<span class="lineNum">      58 </span><span class="lineNoCov">          0 :         (Dtype(this-&gt;iter_) / Dtype(this-&gt;param_.max_iter())),</span>
<span class="lineNum">      59 </span>            :         this-&gt;param_.power());
<span class="lineNum">      60 </span><span class="lineNoCov">          0 :   } else if (lr_policy == &quot;sigmoid&quot;) {</span>
<span class="lineNum">      61 </span><span class="lineNoCov">          0 :     CHECK_GE(this-&gt;param_.gamma(), 0);</span>
<span class="lineNum">      62 </span><span class="lineNoCov">          0 :     CHECK_GT(this-&gt;param_.stepsize(), 0);</span>
<span class="lineNum">      63 </span><span class="lineNoCov">          0 :     rate = this-&gt;param_.base_lr() * (Dtype(1.) /</span>
<span class="lineNum">      64 </span><span class="lineNoCov">          0 :         (Dtype(1.) + exp(-this-&gt;param_.gamma() * (Dtype(this-&gt;iter_) -</span>
<span class="lineNum">      65 </span>            :           Dtype(this-&gt;param_.stepsize())))));
<span class="lineNum">      66 </span>            :   } else {
<span class="lineNum">      67 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Unknown learning rate policy: &quot; &lt;&lt; lr_policy;</span>
<span class="lineNum">      68 </span>            :   }
<span class="lineNum">      69 </span><span class="lineNoCov">          0 :   return rate;</span>
<span class="lineNum">      70 </span>            : }
<span class="lineNum">      71 </span>            : 
<span class="lineNum">      72 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      73 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::PreSolve() {</span>
<span class="lineNum">      74 </span>            :   // Initialize the history
<span class="lineNum">      75 </span>            :   const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; net_params = this-&gt;net_-&gt;learnable_params();
<span class="lineNum">      76 </span><span class="lineNoCov">          0 :   history_.clear();</span>
<span class="lineNum">      77 </span><span class="lineNoCov">          0 :   update_.clear();</span>
<span class="lineNum">      78 </span><span class="lineNoCov">          0 :   temp_.clear();</span>
<span class="lineNum">      79 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; net_params.size(); ++i) {</span>
<span class="lineNum">      80 </span><span class="lineNoCov">          0 :     const vector&lt;int&gt;&amp; shape = net_params[i]-&gt;shape();</span>
<span class="lineNum">      81 </span><span class="lineNoCov">          0 :     history_.push_back(shared_ptr&lt;Blob&lt;Dtype&gt; &gt;(new Blob&lt;Dtype&gt;(shape)));</span>
<span class="lineNum">      82 </span><span class="lineNoCov">          0 :     update_.push_back(shared_ptr&lt;Blob&lt;Dtype&gt; &gt;(new Blob&lt;Dtype&gt;(shape)));</span>
<span class="lineNum">      83 </span><span class="lineNoCov">          0 :     temp_.push_back(shared_ptr&lt;Blob&lt;Dtype&gt; &gt;(new Blob&lt;Dtype&gt;(shape)));</span>
<span class="lineNum">      84 </span>            :   }
<span class="lineNum">      85 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">      86 </span>            : 
<span class="lineNum">      87 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">      88 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::ClipGradients() {</span>
<span class="lineNum">      89 </span><span class="lineNoCov">          0 :   const Dtype clip_gradients = this-&gt;param_.clip_gradients();</span>
<span class="lineNum">      90 </span><span class="lineNoCov">          0 :   if (clip_gradients &lt; 0) { return; }</span>
<span class="lineNum">      91 </span>            :   const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; net_params = this-&gt;net_-&gt;learnable_params();
<span class="lineNum">      92 </span>            :   Dtype sumsq_diff = 0;
<span class="lineNum">      93 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; net_params.size(); ++i) {</span>
<span class="lineNum">      94 </span><span class="lineNoCov">          0 :     sumsq_diff += net_params[i]-&gt;sumsq_diff();</span>
<span class="lineNum">      95 </span>            :   }
<span class="lineNum">      96 </span><span class="lineNoCov">          0 :   const Dtype l2norm_diff = std::sqrt(sumsq_diff);</span>
<span class="lineNum">      97 </span><span class="lineNoCov">          0 :   if (l2norm_diff &gt; clip_gradients) {</span>
<span class="lineNum">      98 </span><span class="lineNoCov">          0 :     Dtype scale_factor = clip_gradients / l2norm_diff;</span>
<span class="lineNum">      99 </span><span class="lineNoCov">          0 :     LOG(INFO) &lt;&lt; &quot;Gradient clipping: scaling down gradients (L2 norm &quot;</span>
<span class="lineNum">     100 </span>            :         &lt;&lt; l2norm_diff &lt;&lt; &quot; &gt; &quot; &lt;&lt; clip_gradients &lt;&lt; &quot;) &quot;
<span class="lineNum">     101 </span>            :         &lt;&lt; &quot;by scale factor &quot; &lt;&lt; scale_factor;
<span class="lineNum">     102 </span><span class="lineNoCov">          0 :     for (int i = 0; i &lt; net_params.size(); ++i) {</span>
<span class="lineNum">     103 </span><span class="lineNoCov">          0 :       net_params[i]-&gt;scale_diff(scale_factor);</span>
<span class="lineNum">     104 </span>            :     }
<span class="lineNum">     105 </span>            :   }
<span class="lineNum">     106 </span>            : }
<span class="lineNum">     107 </span>            : 
<span class="lineNum">     108 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     109 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::ApplyUpdate() {</span>
<span class="lineNum">     110 </span><span class="lineNoCov">          0 :   Dtype rate = GetLearningRate();</span>
<span class="lineNum">     111 </span><span class="lineNoCov">          0 :   if (this-&gt;param_.display() &amp;&amp; this-&gt;iter_ % this-&gt;param_.display() == 0) {</span>
<span class="lineNum">     112 </span><span class="lineNoCov">          0 :     LOG_IF(INFO, Caffe::root_solver()) &lt;&lt; &quot;Iteration &quot; &lt;&lt; this-&gt;iter_</span>
<span class="lineNum">     113 </span>            :         &lt;&lt; &quot;, lr = &quot; &lt;&lt; rate;
<span class="lineNum">     114 </span>            :   }
<span class="lineNum">     115 </span><span class="lineNoCov">          0 :   ClipGradients();</span>
<span class="lineNum">     116 </span><span class="lineNoCov">          0 :   for (int param_id = 0; param_id &lt; this-&gt;net_-&gt;learnable_params().size();</span>
<span class="lineNum">     117 </span>            :        ++param_id) {
<span class="lineNum">     118 </span><span class="lineNoCov">          0 :     Normalize(param_id);</span>
<span class="lineNum">     119 </span><span class="lineNoCov">          0 :     Regularize(param_id);</span>
<span class="lineNum">     120 </span><span class="lineNoCov">          0 :     ComputeUpdateValue(param_id, rate);</span>
<span class="lineNum">     121 </span>            :   }
<span class="lineNum">     122 </span><span class="lineNoCov">          0 :   this-&gt;net_-&gt;Update();</span>
<span class="lineNum">     123 </span>            : 
<span class="lineNum">     124 </span>            :   // Increment the internal iter_ counter -- its value should always indicate
<span class="lineNum">     125 </span>            :   // the number of times the weights have been updated.
<span class="lineNum">     126 </span><span class="lineNoCov">          0 :   ++this-&gt;iter_;</span>
<span class="lineNum">     127 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     128 </span>            : 
<span class="lineNum">     129 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     130 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::Normalize(int param_id) {</span>
<span class="lineNum">     131 </span><span class="lineNoCov">          0 :   if (this-&gt;param_.iter_size() == 1) { return; }</span>
<span class="lineNum">     132 </span>            :   // Scale gradient to counterbalance accumulation.
<span class="lineNum">     133 </span>            :   const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; net_params = this-&gt;net_-&gt;learnable_params();
<span class="lineNum">     134 </span><span class="lineNoCov">          0 :   const Dtype accum_normalization = Dtype(1.) / this-&gt;param_.iter_size();</span>
<span class="lineNum">     135 </span><span class="lineNoCov">          0 :   switch (Caffe::mode()) {</span>
<span class="lineNum">     136 </span>            :   case Caffe::CPU: {
<span class="lineNum">     137 </span><span class="lineNoCov">          0 :     caffe_scal(net_params[param_id]-&gt;count(), accum_normalization,</span>
<span class="lineNum">     138 </span><span class="lineNoCov">          0 :         net_params[param_id]-&gt;mutable_cpu_diff());</span>
<span class="lineNum">     139 </span><span class="lineNoCov">          0 :     break;</span>
<span class="lineNum">     140 </span>            :   }
<span class="lineNum">     141 </span>            :   case Caffe::GPU: {
<span class="lineNum">     142 </span>            : #ifndef CPU_ONLY
<span class="lineNum">     143 </span>            :     caffe_gpu_scal(net_params[param_id]-&gt;count(), accum_normalization,
<span class="lineNum">     144 </span>            :         net_params[param_id]-&gt;mutable_gpu_diff());
<span class="lineNum">     145 </span>            : #else
<span class="lineNum">     146 </span><span class="lineNoCov">          0 :     NO_GPU;</span>
<span class="lineNum">     147 </span>            : #endif
<span class="lineNum">     148 </span>            :     break;
<span class="lineNum">     149 </span>            :   }
<span class="lineNum">     150 </span>            :   default:
<span class="lineNum">     151 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Unknown caffe mode: &quot; &lt;&lt; Caffe::mode();</span>
<span class="lineNum">     152 </span>            :   }
<span class="lineNum">     153 </span>            : }
<span class="lineNum">     154 </span>            : 
<span class="lineNum">     155 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     156 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::Regularize(int param_id) {</span>
<span class="lineNum">     157 </span>            :   const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; net_params = this-&gt;net_-&gt;learnable_params();
<span class="lineNum">     158 </span>            :   const vector&lt;float&gt;&amp; net_params_weight_decay =
<span class="lineNum">     159 </span>            :       this-&gt;net_-&gt;params_weight_decay();
<span class="lineNum">     160 </span><span class="lineNoCov">          0 :   Dtype weight_decay = this-&gt;param_.weight_decay();</span>
<span class="lineNum">     161 </span>            :   string regularization_type = this-&gt;param_.regularization_type();
<span class="lineNum">     162 </span><span class="lineNoCov">          0 :   Dtype local_decay = weight_decay * net_params_weight_decay[param_id];</span>
<span class="lineNum">     163 </span><span class="lineNoCov">          0 :   switch (Caffe::mode()) {</span>
<span class="lineNum">     164 </span>            :   case Caffe::CPU: {
<span class="lineNum">     165 </span><span class="lineNoCov">          0 :     if (local_decay) {</span>
<span class="lineNum">     166 </span><span class="lineNoCov">          0 :       if (regularization_type == &quot;L2&quot;) {</span>
<span class="lineNum">     167 </span>            :         // add weight decay
<span class="lineNum">     168 </span><span class="lineNoCov">          0 :         caffe_axpy(net_params[param_id]-&gt;count(),</span>
<span class="lineNum">     169 </span>            :             local_decay,
<span class="lineNum">     170 </span>            :             net_params[param_id]-&gt;cpu_data(),
<span class="lineNum">     171 </span>            :             net_params[param_id]-&gt;mutable_cpu_diff());
<span class="lineNum">     172 </span><span class="lineNoCov">          0 :       } else if (regularization_type == &quot;L1&quot;) {</span>
<span class="lineNum">     173 </span><span class="lineNoCov">          0 :         caffe_cpu_sign(net_params[param_id]-&gt;count(),</span>
<span class="lineNum">     174 </span>            :             net_params[param_id]-&gt;cpu_data(),
<span class="lineNum">     175 </span>            :             temp_[param_id]-&gt;mutable_cpu_data());
<span class="lineNum">     176 </span><span class="lineNoCov">          0 :         caffe_axpy(net_params[param_id]-&gt;count(),</span>
<span class="lineNum">     177 </span>            :             local_decay,
<span class="lineNum">     178 </span>            :             temp_[param_id]-&gt;cpu_data(),
<span class="lineNum">     179 </span>            :             net_params[param_id]-&gt;mutable_cpu_diff());
<span class="lineNum">     180 </span>            :       } else {
<span class="lineNum">     181 </span><span class="lineNoCov">          0 :         LOG(FATAL) &lt;&lt; &quot;Unknown regularization type: &quot; &lt;&lt; regularization_type;</span>
<span class="lineNum">     182 </span>            :       }
<span class="lineNum">     183 </span>            :     }
<span class="lineNum">     184 </span>            :     break;
<span class="lineNum">     185 </span>            :   }
<span class="lineNum">     186 </span>            :   case Caffe::GPU: {
<span class="lineNum">     187 </span>            : #ifndef CPU_ONLY
<span class="lineNum">     188 </span>            :     if (local_decay) {
<span class="lineNum">     189 </span>            :       if (regularization_type == &quot;L2&quot;) {
<span class="lineNum">     190 </span>            :         // add weight decay
<span class="lineNum">     191 </span>            :         caffe_gpu_axpy(net_params[param_id]-&gt;count(),
<span class="lineNum">     192 </span>            :             local_decay,
<span class="lineNum">     193 </span>            :             net_params[param_id]-&gt;gpu_data(),
<span class="lineNum">     194 </span>            :             net_params[param_id]-&gt;mutable_gpu_diff());
<span class="lineNum">     195 </span>            :       } else if (regularization_type == &quot;L1&quot;) {
<span class="lineNum">     196 </span>            :         caffe_gpu_sign(net_params[param_id]-&gt;count(),
<span class="lineNum">     197 </span>            :             net_params[param_id]-&gt;gpu_data(),
<span class="lineNum">     198 </span>            :             temp_[param_id]-&gt;mutable_gpu_data());
<span class="lineNum">     199 </span>            :         caffe_gpu_axpy(net_params[param_id]-&gt;count(),
<span class="lineNum">     200 </span>            :             local_decay,
<span class="lineNum">     201 </span>            :             temp_[param_id]-&gt;gpu_data(),
<span class="lineNum">     202 </span>            :             net_params[param_id]-&gt;mutable_gpu_diff());
<span class="lineNum">     203 </span>            :       } else {
<span class="lineNum">     204 </span>            :         LOG(FATAL) &lt;&lt; &quot;Unknown regularization type: &quot; &lt;&lt; regularization_type;
<span class="lineNum">     205 </span>            :       }
<span class="lineNum">     206 </span>            :     }
<span class="lineNum">     207 </span>            : #else
<span class="lineNum">     208 </span><span class="lineNoCov">          0 :     NO_GPU;</span>
<span class="lineNum">     209 </span>            : #endif
<span class="lineNum">     210 </span>            :     break;
<span class="lineNum">     211 </span>            :   }
<span class="lineNum">     212 </span>            :   default:
<span class="lineNum">     213 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Unknown caffe mode: &quot; &lt;&lt; Caffe::mode();</span>
<span class="lineNum">     214 </span>            :   }
<span class="lineNum">     215 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     216 </span>            : 
<span class="lineNum">     217 </span>            : #ifndef CPU_ONLY
<span class="lineNum">     218 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     219 </span>            : void sgd_update_gpu(int N, Dtype* g, Dtype* h, Dtype momentum,
<span class="lineNum">     220 </span>            :     Dtype local_rate);
<span class="lineNum">     221 </span>            : #endif
<span class="lineNum">     222 </span>            : 
<span class="lineNum">     223 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     224 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::ComputeUpdateValue(int param_id, Dtype rate) {</span>
<span class="lineNum">     225 </span>            :   const vector&lt;Blob&lt;Dtype&gt;*&gt;&amp; net_params = this-&gt;net_-&gt;learnable_params();
<span class="lineNum">     226 </span>            :   const vector&lt;float&gt;&amp; net_params_lr = this-&gt;net_-&gt;params_lr();
<span class="lineNum">     227 </span><span class="lineNoCov">          0 :   Dtype momentum = this-&gt;param_.momentum();</span>
<span class="lineNum">     228 </span><span class="lineNoCov">          0 :   Dtype local_rate = rate * net_params_lr[param_id];</span>
<span class="lineNum">     229 </span>            :   // Compute the update to history, then copy it to the parameter diff.
<span class="lineNum">     230 </span><span class="lineNoCov">          0 :   switch (Caffe::mode()) {</span>
<span class="lineNum">     231 </span>            :   case Caffe::CPU: {
<span class="lineNum">     232 </span><span class="lineNoCov">          0 :     caffe_cpu_axpby(net_params[param_id]-&gt;count(), local_rate,</span>
<span class="lineNum">     233 </span>            :               net_params[param_id]-&gt;cpu_diff(), momentum,
<span class="lineNum">     234 </span>            :               history_[param_id]-&gt;mutable_cpu_data());
<span class="lineNum">     235 </span><span class="lineNoCov">          0 :     caffe_copy(net_params[param_id]-&gt;count(),</span>
<span class="lineNum">     236 </span>            :         history_[param_id]-&gt;cpu_data(),
<span class="lineNum">     237 </span>            :         net_params[param_id]-&gt;mutable_cpu_diff());
<span class="lineNum">     238 </span>            :     break;
<span class="lineNum">     239 </span>            :   }
<span class="lineNum">     240 </span>            :   case Caffe::GPU: {
<span class="lineNum">     241 </span>            : #ifndef CPU_ONLY
<span class="lineNum">     242 </span>            :     sgd_update_gpu(net_params[param_id]-&gt;count(),
<span class="lineNum">     243 </span>            :         net_params[param_id]-&gt;mutable_gpu_diff(),
<span class="lineNum">     244 </span>            :         history_[param_id]-&gt;mutable_gpu_data(),
<span class="lineNum">     245 </span>            :         momentum, local_rate);
<span class="lineNum">     246 </span>            : #else
<span class="lineNum">     247 </span><span class="lineNoCov">          0 :     NO_GPU;</span>
<span class="lineNum">     248 </span>            : #endif
<span class="lineNum">     249 </span>            :     break;
<span class="lineNum">     250 </span>            :   }
<span class="lineNum">     251 </span>            :   default:
<span class="lineNum">     252 </span><span class="lineNoCov">          0 :     LOG(FATAL) &lt;&lt; &quot;Unknown caffe mode: &quot; &lt;&lt; Caffe::mode();</span>
<span class="lineNum">     253 </span>            :   }
<span class="lineNum">     254 </span><span class="lineNoCov">          0 : }</span>
<a name="255"><span class="lineNum">     255 </span>            : </a>
<span class="lineNum">     256 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     257 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::SnapshotSolverState(const string&amp; model_filename) {</span>
<span class="lineNum">     258 </span><span class="lineNoCov">          0 :   switch (this-&gt;param_.snapshot_format()) {</span>
<span class="lineNum">     259 </span>            :     case caffe::SolverParameter_SnapshotFormat_BINARYPROTO:
<span class="lineNum">     260 </span><span class="lineNoCov">          0 :       SnapshotSolverStateToBinaryProto(model_filename);</span>
<span class="lineNum">     261 </span><span class="lineNoCov">          0 :       break;</span>
<span class="lineNum">     262 </span>            :     case caffe::SolverParameter_SnapshotFormat_HDF5:
<span class="lineNum">     263 </span><span class="lineNoCov">          0 :       SnapshotSolverStateToHDF5(model_filename);</span>
<span class="lineNum">     264 </span><span class="lineNoCov">          0 :       break;</span>
<span class="lineNum">     265 </span>            :     default:
<span class="lineNum">     266 </span><span class="lineNoCov">          0 :       LOG(FATAL) &lt;&lt; &quot;Unsupported snapshot format.&quot;;</span>
<span class="lineNum">     267 </span>            :   }
<span class="lineNum">     268 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     269 </span>            : 
<span class="lineNum">     270 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     271 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::SnapshotSolverStateToBinaryProto(</span>
<span class="lineNum">     272 </span>            :     const string&amp; model_filename) {
<span class="lineNum">     273 </span><span class="lineNoCov">          0 :   SolverState state;</span>
<span class="lineNum">     274 </span><span class="lineNoCov">          0 :   state.set_iter(this-&gt;iter_);</span>
<span class="lineNum">     275 </span><span class="lineNoCov">          0 :   state.set_learned_net(model_filename);</span>
<span class="lineNum">     276 </span><span class="lineNoCov">          0 :   state.set_current_step(this-&gt;current_step_);</span>
<span class="lineNum">     277 </span>            :   state.clear_history();
<span class="lineNum">     278 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; history_.size(); ++i) {</span>
<span class="lineNum">     279 </span>            :     // Add history
<span class="lineNum">     280 </span>            :     BlobProto* history_blob = state.add_history();
<span class="lineNum">     281 </span><span class="lineNoCov">          0 :     history_[i]-&gt;ToProto(history_blob);</span>
<span class="lineNum">     282 </span>            :   }
<span class="lineNum">     283 </span><span class="lineNoCov">          0 :   string snapshot_filename = Solver&lt;Dtype&gt;::SnapshotFilename(&quot;.solverstate&quot;);</span>
<span class="lineNum">     284 </span><span class="lineNoCov">          0 :   LOG(INFO)</span>
<span class="lineNum">     285 </span>            :     &lt;&lt; &quot;Snapshotting solver state to binary proto file &quot; &lt;&lt; snapshot_filename;
<span class="lineNum">     286 </span><span class="lineNoCov">          0 :   WriteProtoToBinaryFile(state, snapshot_filename.c_str());</span>
<span class="lineNum">     287 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     288 </span>            : 
<span class="lineNum">     289 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     290 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::SnapshotSolverStateToHDF5(</span>
<span class="lineNum">     291 </span>            :     const string&amp; model_filename) {
<span class="lineNum">     292 </span>            : // This code is taken from https://github.com/sh1r0/caffe-android-lib
<span class="lineNum">     293 </span>            : #ifdef USE_HDF5
<span class="lineNum">     294 </span>            :   string snapshot_filename =
<span class="lineNum">     295 </span><span class="lineNoCov">          0 :       Solver&lt;Dtype&gt;::SnapshotFilename(&quot;.solverstate.h5&quot;);</span>
<span class="lineNum">     296 </span><span class="lineNoCov">          0 :   LOG(INFO) &lt;&lt; &quot;Snapshotting solver state to HDF5 file &quot; &lt;&lt; snapshot_filename;</span>
<span class="lineNum">     297 </span><span class="lineNoCov">          0 :   hid_t file_hid = H5Fcreate(snapshot_filename.c_str(), H5F_ACC_TRUNC,</span>
<span class="lineNum">     298 </span><span class="lineNoCov">          0 :       H5P_DEFAULT, H5P_DEFAULT);</span>
<span class="lineNum">     299 </span><span class="lineNoCov">          0 :   CHECK_GE(file_hid, 0)</span>
<span class="lineNum">     300 </span>            :       &lt;&lt; &quot;Couldn't open &quot; &lt;&lt; snapshot_filename &lt;&lt; &quot; to save solver state.&quot;;
<span class="lineNum">     301 </span><span class="lineNoCov">          0 :   hdf5_save_int(file_hid, &quot;iter&quot;, this-&gt;iter_);</span>
<span class="lineNum">     302 </span><span class="lineNoCov">          0 :   hdf5_save_string(file_hid, &quot;learned_net&quot;, model_filename);</span>
<span class="lineNum">     303 </span><span class="lineNoCov">          0 :   hdf5_save_int(file_hid, &quot;current_step&quot;, this-&gt;current_step_);</span>
<span class="lineNum">     304 </span>            :   hid_t history_hid = H5Gcreate2(file_hid, &quot;history&quot;, H5P_DEFAULT, H5P_DEFAULT,
<span class="lineNum">     305 </span><span class="lineNoCov">          0 :       H5P_DEFAULT);</span>
<span class="lineNum">     306 </span><span class="lineNoCov">          0 :   CHECK_GE(history_hid, 0)</span>
<span class="lineNum">     307 </span>            :       &lt;&lt; &quot;Error saving solver state to &quot; &lt;&lt; snapshot_filename &lt;&lt; &quot;.&quot;;
<span class="lineNum">     308 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; history_.size(); ++i) {</span>
<span class="lineNum">     309 </span><span class="lineNoCov">          0 :     ostringstream oss;</span>
<span class="lineNum">     310 </span><span class="lineNoCov">          0 :     oss &lt;&lt; i;</span>
<span class="lineNum">     311 </span><span class="lineNoCov">          0 :     hdf5_save_nd_dataset&lt;Dtype&gt;(history_hid, oss.str(), *history_[i]);</span>
<span class="lineNum">     312 </span>            :   }
<span class="lineNum">     313 </span><span class="lineNoCov">          0 :   H5Gclose(history_hid);</span>
<span class="lineNum">     314 </span><span class="lineNoCov">          0 :   H5Fclose(file_hid);</span>
<span class="lineNum">     315 </span>            : // This code is taken from https://github.com/sh1r0/caffe-android-lib
<span class="lineNum">     316 </span>            : #else
<span class="lineNum">     317 </span>            :   LOG(FATAL) &lt;&lt; &quot;SnapshotSolverStateToHDF5 requires hdf5;&quot;
<span class="lineNum">     318 </span>            :              &lt;&lt; &quot; compile with USE_HDF5.&quot;;
<span class="lineNum">     319 </span>            : #endif  // USE_HDF5
<span class="lineNum">     320 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     321 </span>            : 
<span class="lineNum">     322 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     323 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::RestoreSolverStateFromBinaryProto(</span>
<span class="lineNum">     324 </span>            :     const string&amp; state_file) {
<span class="lineNum">     325 </span><span class="lineNoCov">          0 :   SolverState state;</span>
<span class="lineNum">     326 </span>            :   ReadProtoFromBinaryFile(state_file, &amp;state);
<span class="lineNum">     327 </span><span class="lineNoCov">          0 :   this-&gt;iter_ = state.iter();</span>
<span class="lineNum">     328 </span><span class="lineNoCov">          0 :   if (state.has_learned_net()) {</span>
<span class="lineNum">     329 </span><span class="lineNoCov">          0 :     NetParameter net_param;</span>
<span class="lineNum">     330 </span><span class="lineNoCov">          0 :     ReadNetParamsFromBinaryFileOrDie(state.learned_net().c_str(), &amp;net_param);</span>
<span class="lineNum">     331 </span><span class="lineNoCov">          0 :     this-&gt;net_-&gt;CopyTrainedLayersFrom(net_param);</span>
<span class="lineNum">     332 </span>            :   }
<span class="lineNum">     333 </span><span class="lineNoCov">          0 :   this-&gt;current_step_ = state.current_step();</span>
<span class="lineNum">     334 </span><span class="lineNoCov">          0 :   CHECK_EQ(state.history_size(), history_.size())</span>
<span class="lineNum">     335 </span>            :       &lt;&lt; &quot;Incorrect length of history blobs.&quot;;
<span class="lineNum">     336 </span><span class="lineNoCov">          0 :   LOG(INFO) &lt;&lt; &quot;SGDSolver: restoring history&quot;;</span>
<span class="lineNum">     337 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; history_.size(); ++i) {</span>
<span class="lineNum">     338 </span><span class="lineNoCov">          0 :     history_[i]-&gt;FromProto(state.history(i));</span>
<span class="lineNum">     339 </span>            :   }
<span class="lineNum">     340 </span><span class="lineNoCov">          0 : }</span>
<span class="lineNum">     341 </span>            : 
<span class="lineNum">     342 </span>            : template &lt;typename Dtype&gt;
<span class="lineNum">     343 </span><span class="lineNoCov">          0 : void SGDSolver&lt;Dtype&gt;::RestoreSolverStateFromHDF5(const string&amp; state_file) {</span>
<span class="lineNum">     344 </span>            : #ifdef USE_HDF5
<span class="lineNum">     345 </span><span class="lineNoCov">          0 :   hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);</span>
<span class="lineNum">     346 </span><span class="lineNoCov">          0 :   CHECK_GE(file_hid, 0) &lt;&lt; &quot;Couldn't open solver state file &quot; &lt;&lt; state_file;</span>
<span class="lineNum">     347 </span><span class="lineNoCov">          0 :   this-&gt;iter_ = hdf5_load_int(file_hid, &quot;iter&quot;);</span>
<span class="lineNum">     348 </span><span class="lineNoCov">          0 :   if (H5LTfind_dataset(file_hid, &quot;learned_net&quot;)) {</span>
<span class="lineNum">     349 </span><span class="lineNoCov">          0 :     string learned_net = hdf5_load_string(file_hid, &quot;learned_net&quot;);</span>
<span class="lineNum">     350 </span><span class="lineNoCov">          0 :     this-&gt;net_-&gt;CopyTrainedLayersFrom(learned_net);</span>
<span class="lineNum">     351 </span>            :   }
<span class="lineNum">     352 </span><span class="lineNoCov">          0 :   this-&gt;current_step_ = hdf5_load_int(file_hid, &quot;current_step&quot;);</span>
<span class="lineNum">     353 </span><span class="lineNoCov">          0 :   hid_t history_hid = H5Gopen2(file_hid, &quot;history&quot;, H5P_DEFAULT);</span>
<span class="lineNum">     354 </span><span class="lineNoCov">          0 :   CHECK_GE(history_hid, 0) &lt;&lt; &quot;Error reading history from &quot; &lt;&lt; state_file;</span>
<span class="lineNum">     355 </span><span class="lineNoCov">          0 :   int state_history_size = hdf5_get_num_links(history_hid);</span>
<span class="lineNum">     356 </span><span class="lineNoCov">          0 :   CHECK_EQ(state_history_size, history_.size())</span>
<span class="lineNum">     357 </span>            :       &lt;&lt; &quot;Incorrect length of history blobs.&quot;;
<span class="lineNum">     358 </span><span class="lineNoCov">          0 :   for (int i = 0; i &lt; history_.size(); ++i) {</span>
<span class="lineNum">     359 </span><span class="lineNoCov">          0 :     ostringstream oss;</span>
<span class="lineNum">     360 </span><span class="lineNoCov">          0 :     oss &lt;&lt; i;</span>
<span class="lineNum">     361 </span><span class="lineNoCov">          0 :     hdf5_load_nd_dataset&lt;Dtype&gt;(history_hid, oss.str().c_str(), 0,</span>
<span class="lineNum">     362 </span>            :                                 kMaxBlobAxes, history_[i].get());
<span class="lineNum">     363 </span>            :   }
<span class="lineNum">     364 </span><span class="lineNoCov">          0 :   H5Gclose(history_hid);</span>
<span class="lineNum">     365 </span><span class="lineNoCov">          0 :   H5Fclose(file_hid);</span>
<span class="lineNum">     366 </span>            : #else
<span class="lineNum">     367 </span>            :   LOG(FATAL) &lt;&lt; &quot;RestoreSolverStateFromHDF5 requires hdf5;&quot;
<span class="lineNum">     368 </span>            :              &lt;&lt; &quot; compile with USE_HDF5.&quot;;
<span class="lineNum">     369 </span>            : #endif  // USE_HDF5
<span class="lineNum">     370 </span><span class="lineNoCov">          0 : }</span>
<a name="371"><span class="lineNum">     371 </span>            : </a>
<span class="lineNum">     372 </span>            : INSTANTIATE_CLASS(SGDSolver);
<a name="373"><span class="lineNum">     373 </span><span class="lineCov">          3 : REGISTER_SOLVER_CLASS(SGD);</span></a>
<span class="lineNum">     374 </span>            : 
<span class="lineNum">     375 </span><span class="lineCov">          3 : }  // namespace caffe</span>
</pre>
      </td>
    </tr>
  </table>
  <br>

  <table width="100%" border=0 cellspacing=0 cellpadding=0>
    <tr><td class="ruler"><img src="../../../glass.png" width=3 height=3 alt=""></td></tr>
    <tr><td class="versionInfo">Generated by: <a href="http://ltp.sourceforge.net/coverage/lcov.php" target="_parent">LCOV version 1.12</a></td></tr>
  </table>
  <br>

</body>
</html>
